{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.13.1\n",
      "sys.version_info(major=3, minor=7, micro=3, releaselevel='final', serial=0)\n",
      "matplotlib 3.0.3\n",
      "numpy 1.16.2\n",
      "pandas 0.24.2\n",
      "sklearn 0.20.3\n",
      "tensorflow 1.13.1\n",
      "tensorflow._api.v1.keras 2.2.4-tf\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow import keras\n",
    "\n",
    "print(tf.__version__)\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, tf, keras:\n",
    "    print(module.__name__, module.__version__)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 28, 28) (5000,)\n",
      "(55000, 28, 28) (55000,)\n",
      "(10000, 28, 28) (10000,)\n"
     ]
    }
   ],
   "source": [
    "fashion_mnist = keras.datasets.fashion_mnist\n",
    "(x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()\n",
    "x_valid, x_train = x_train_all[:5000], x_train_all[5000:]\n",
    "y_valid, y_train = y_train_all[:5000], y_train_all[5000:]\n",
    "\n",
    "print(x_valid.shape, y_valid.shape)\n",
    "print(x_train.shape, y_train.shape)\n",
    "print(x_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "255 0\n"
     ]
    }
   ],
   "source": [
    "print(np.max(x_train), np.min(x_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x = (x - u) / std\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "# x_train: [None, 28, 28] -> [None, 784]\n",
    "x_train_scaled = scaler.fit_transform(\n",
    "    x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28 * 28)\n",
    "x_valid_scaled = scaler.transform(\n",
    "    x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28 * 28)\n",
    "x_test_scaled = scaler.transform(\n",
    "    x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28 * 28)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.023144 -0.8105139\n"
     ]
    }
   ],
   "source": [
    "print(np.max(x_train_scaled), np.min(x_train_scaled))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-6-76fc28fa9449>:11: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:From /Users/zhangyx/workspace/environments/tf_py3/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /Users/zhangyx/workspace/environments/tf_py3/lib/python3.7/site-packages/tensorflow/python/ops/losses/losses_impl.py:209: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    }
   ],
   "source": [
    "hidden_units = [100, 100]\n",
    "class_num = 10\n",
    "\n",
    "x = tf.placeholder(tf.float32, [None, 28 * 28])\n",
    "y = tf.placeholder(tf.int64, [None])\n",
    "\n",
    "input_for_next_layer = x\n",
    "for hidden_unit in hidden_units:\n",
    "    input_for_next_layer = tf.layers.dense(input_for_next_layer,\n",
    "                                           hidden_unit,\n",
    "                                           activation=tf.nn.relu)\n",
    "logits = tf.layers.dense(input_for_next_layer,\n",
    "                         class_num)\n",
    "# last_hidden_output * W(logits) -> softmax -> prob\n",
    "# 1. logit -> softmax -> prob\n",
    "# 2. labels -> one_hot\n",
    "# 3. calculate cross entropy\n",
    "loss = tf.losses.sparse_softmax_cross_entropy(labels = y,\n",
    "                                              logits = logits)\n",
    "# get accuracy.\n",
    "prediction = tf.argmax(logits, 1)\n",
    "correct_prediction = tf.equal(prediction, y)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float64))\n",
    "\n",
    "train_op = tf.train.AdamOptimizer(1e-3).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"Placeholder:0\", shape=(?, 784), dtype=float32)\n",
      "Tensor(\"dense_2/BiasAdd:0\", shape=(?, 10), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(x)\n",
    "print(logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] epoch: 0, step: 2749, loss: 0.29409, accuracy: 0.85\t[Valid] acc: 0.86\n",
      "[Train] epoch: 1, step: 2749, loss: 0.23783, accuracy: 0.90\t[Valid] acc: 0.87\n",
      "[Train] epoch: 2, step: 2749, loss: 0.16557, accuracy: 0.90\t[Valid] acc: 0.87\n",
      "[Train] epoch: 3, step: 2749, loss: 0.16518, accuracy: 0.85\t[Valid] acc: 0.88\n",
      "[Train] epoch: 4, step: 2749, loss: 0.18367, accuracy: 0.90\t[Valid] acc: 0.88\n",
      "[Train] epoch: 5, step: 2749, loss: 0.18992, accuracy: 0.95\t[Valid] acc: 0.88\n",
      "[Train] epoch: 6, step: 2749, loss: 0.13003, accuracy: 0.95\t[Valid] acc: 0.88\n",
      "[Train] epoch: 7, step: 2749, loss: 0.19065, accuracy: 0.95\t[Valid] acc: 0.89\n",
      "[Train] epoch: 8, step: 2749, loss: 0.12379, accuracy: 0.90\t[Valid] acc: 0.88\n",
      "[Train] epoch: 9, step: 2749, loss: 0.11657, accuracy: 0.95\t[Valid] acc: 0.89\n"
     ]
    }
   ],
   "source": [
    "# session\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "batch_size = 20\n",
    "epochs = 10\n",
    "train_steps_per_epoch = x_train.shape[0] // batch_size\n",
    "valid_steps = x_valid.shape[0] // batch_size\n",
    "\n",
    "def eval_with_sess(sess, x, y, accuracy, images, labels, batch_size):\n",
    "    eval_steps = images.shape[0] // batch_size\n",
    "    eval_accuracies = []\n",
    "    for step in range(eval_steps):\n",
    "        batch_data = images[step * batch_size : (step+1) * batch_size]\n",
    "        batch_label = labels[step * batch_size : (step+1) * batch_size]\n",
    "        accuracy_val = sess.run(accuracy,\n",
    "                                feed_dict = {\n",
    "                                    x: batch_data,\n",
    "                                    y: batch_label\n",
    "                                })\n",
    "        eval_accuracies.append(accuracy_val)\n",
    "    return np.mean(eval_accuracies)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for epoch in range(epochs):\n",
    "        for step in range(train_steps_per_epoch):\n",
    "            batch_data = x_train_scaled[\n",
    "                step * batch_size : (step+1) * batch_size]\n",
    "            batch_label = y_train[\n",
    "                step * batch_size : (step+1) * batch_size]\n",
    "            loss_val, accuracy_val, _ = sess.run(\n",
    "                [loss, accuracy, train_op],\n",
    "                feed_dict = {\n",
    "                    x: batch_data,\n",
    "                    y: batch_label\n",
    "                })\n",
    "            print('\\r[Train] epoch: %d, step: %d, loss: %3.5f, accuracy: %2.2f' % (\n",
    "                epoch, step, loss_val, accuracy_val), end=\"\")\n",
    "        valid_accuracy = eval_with_sess(sess, x, y, accuracy,\n",
    "                                        x_valid_scaled, y_valid,\n",
    "                                        batch_size)\n",
    "        print(\"\\t[Valid] acc: %2.2f\" % (valid_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
